Code
# Following https://oliviergimenez.github.io/bayesian-cr-workshop/worksheets/4_demo.html
#
load('./models/cmrNN_OB/tt/runsOut/tt_NN_OB_mostRecent.RData')
out_NN_OB <- d
#Input data
eh <- tar_read(eh_OB_2002_2014_target)Simple CJS models to get phi and p estimates to compare with Neural Network CMR
In all the models below, 1 = not observed and 2 = observed in the input encounter data.
Encounter data are available here in the eh.csv file.
Model with phi and p for each age-in-samples (column in the encounter history file)
Probability of survival (phi) model structure:
phi[t] <- betaInt[t]
Probability of capture (p) model structure:
p[t] <- betaP[t]
Model code is in ./models/cmrNN_OB/tt/modelCMR_tt_NN_OB_functionsToSource.R
Model is run ‘by hand’ in ./models/cmrNN_OB/tt/modelCMR_tt_NN_OB_makeFile.R
Not using this: Targets are loaded in R/modelCMR_tt_NN_OB.R and saved as modelCMR_tt_NN_OB_target
Model runs:
tt_NN_OB_2023-06-14 23.RData: preliminary run only tracing phi and p
tt_NN_OB_2023-06-15 08.RData: tracing phi and p and z [most recent]
# Following https://oliviergimenez.github.io/bayesian-cr-workshop/worksheets/4_demo.html
#
load('./models/cmrNN_OB/tt/runsOut/tt_NN_OB_mostRecent.RData')
out_NN_OB <- d
#Input data
eh <- tar_read(eh_OB_2002_2014_target)In the model code, a value of 1 for z or in gamma or omega signifies the individual is alive and a value of 2 signifies the individual is dead. y[i,j] is the observed data (encounter history file).
out_NN_OB$modelCode{
delta[1] <- 1
delta[2] <- 0
for (t in 1:(T - 1)) {
phi[t] ~ dunif(0, 1)
gamma[1, 1, t] <- phi[t]
gamma[1, 2, t] <- 1 - phi[t]
gamma[2, 1, t] <- 0
gamma[2, 2, t] <- 1
p[t] ~ dunif(0, 1)
omega[1, 1, t] <- 1 - p[t]
omega[1, 2, t] <- p[t]
omega[2, 1, t] <- 1
omega[2, 2, t] <- 0
}
for (i in 1:N) {
z[i, first[i]] ~ dcat(delta[1:2])
for (j in first[i]:(last[i] - 1)) {
z[i, j + 1] ~ dcat(gamma[z[i, j], 1:2, j])
y[i, j + 1] ~ dcat(omega[z[i, j + 1], 1:2, j])
}
}
}
| nIter | nBurnin | nChains | thinRate |
|---|---|---|---|
| 5000 | 2000 | 2 | 5 |
[1] “Run time = 33.374 mins”
priors <- runif(out_NN_OB$runData$nIter * out_NN_OB$runData$nChains, 0, 1)
MCMCtrace(object = out_NN_OB$mcmc,
#ISB = FALSE,
#exact = TRUE,
params = c("phi", "p"),
pdf = FALSE,
priors = priors)







MCMCplot(object = out_NN_OB$mcmc, params = c("phi"))
MCMCplot(object = out_NN_OB$mcmc, params = c("p"))
Summary values
#|
(summary_tt <- MCMCsummary(object = out_NN_OB$mcmc, params = c("phi", "p"), round = 3) %>%
rownames_to_column(var = "var")) var mean sd 2.5% 50% 97.5% Rhat n.eff
1 phi[1] 0.812 0.022 0.769 0.812 0.854 1.01 290
2 phi[2] 0.810 0.025 0.762 0.811 0.859 1.00 218
3 phi[3] 0.749 0.021 0.708 0.749 0.792 1.01 230
4 phi[4] 0.647 0.021 0.606 0.647 0.689 1.00 361
5 phi[5] 0.640 0.026 0.587 0.640 0.687 1.00 258
6 phi[6] 0.691 0.035 0.629 0.690 0.764 1.05 206
7 phi[7] 0.830 0.051 0.736 0.827 0.932 1.02 100
8 phi[8] 0.330 0.032 0.271 0.330 0.393 1.00 312
9 phi[9] 0.669 0.079 0.528 0.665 0.837 1.01 200
10 phi[10] 0.560 0.090 0.401 0.556 0.754 1.08 188
11 phi[11] 0.684 0.151 0.412 0.679 0.958 1.00 103
12 p[1] 0.597 0.022 0.555 0.597 0.638 1.00 449
13 p[2] 0.448 0.019 0.410 0.447 0.487 1.01 512
14 p[3] 0.707 0.020 0.669 0.707 0.746 1.00 441
15 p[4] 0.659 0.022 0.615 0.659 0.701 1.01 452
16 p[5] 0.634 0.025 0.587 0.634 0.682 1.01 457
17 p[6] 0.499 0.028 0.447 0.500 0.554 1.01 380
18 p[7] 0.701 0.041 0.616 0.701 0.780 1.00 156
19 p[8] 0.783 0.048 0.682 0.788 0.869 1.00 401
20 p[9] 0.605 0.072 0.458 0.608 0.736 1.03 325
21 p[10] 0.692 0.088 0.512 0.695 0.849 1.03 312
22 p[11] 0.733 0.155 0.438 0.727 0.989 1.00 123
# To get the mMCMCSummaryRMNA funcion which I adapted to deal with NAs
source('./models/cmrNN_OB/tt/modelCMR_tt_NN_OB_functionsToSource.R')
summary_tt_z0 <- MCMCSummaryRMNA(object = out_NN_OB$mcmc, params = c("z"), round = 3) %>%
rownames_to_column(var = "var") |>
mutate(
ind0 = str_match(var, "([0-9]+), ([0-9]+)")[,2],
t0 = str_match(var, "([0-9]+), ([0-9]+)")[,3],
ind = as.numeric(ind0),
t = as.numeric(t0)
) |>
dplyr::select(-c(t0, ind0))Add other variables to summary values
ehLong <-
eh$eh |>
as.data.frame() |>
rownames_to_column("ind0") |>
pivot_longer(starts_with("ais")) |>
mutate(
t = as.numeric(str_match(name, "([0-9]+)")[,1]),
enc = value,
ind = as.numeric(ind0)
) |>
dplyr::select(-c(name, value, ind0))
firstLong <- eh$first |>
as_tibble() |>
rownames_to_column("ind") |>
mutate(ind = as.numeric(ind)) |>
rename(first = value)
lastLong <- eh$last |>
as_tibble() |>
rownames_to_column("ind") |>
mutate(ind = as.numeric(ind)) |>
rename(last = value)
cohortLong <- eh$cohort |>
as_tibble() |>
rownames_to_column("ind") |>
mutate(ind = as.numeric(ind))
summary_tt_z <- summary_tt_z0 |>
left_join(ehLong) |>
mutate(
meanM1 = mean - 1,
pSurv = ifelse(meanM1 == 0, 1, 1 - meanM1),
enc8 = ifelse(enc == 1, 0.8, 0)
) |>
left_join(firstLong) |>
left_join(lastLong) |>
left_join(cohortLong)ojs_define(numTags_tt_OJS = dim(eh$tags)[1])
ojs_define(summary_tt_z_OJS = (summary_tt_z))Select one or more individuals :
viewof selectInd_tt = Inputs.select([...Array(numTags_tt_OJS).keys()], {
label: "Which fish?",
value: 1,
step: 1,
multiple: true
})
summary_tt_z_OJS_T = transpose(summary_tt_z_OJS)
summary_tt_z_OJS_T_selected = summary_tt_z_OJS_T.filter((d) =>
selectInd_tt.includes(d.ind)
)Black dots/line is estimated probability of survival and orange dots are encountered yes (y = 0.8)/no (y = 0). The green line is the first observation and the red line is the last possible observation. The number in the upper right corner of each panel is the fish’s cohort.
Plot.plot({
width: width,
height: 350,
inset: 10,
color: {
scheme: "greys"
},
x: { label: "Age/season combination" },
y: { label: "Probability of survival" },
marks: [
Plot.frame(),
Plot.dot(summary_tt_z_OJS_T_selected, {
x: "t",
y: "pSurv"
}),
Plot.dot(summary_tt_z_OJS_T_selected, {
x: "t",
y: "enc8",
fill: "orange"
}),
Plot.line(summary_tt_z_OJS_T_selected, {
x: "t",
y: "pSurv"
}),
Plot.ruleX(summary_tt_z_OJS_T_selected, {
x: "first",
y: 1,
"stroke": "green"
}),
Plot.ruleX(summary_tt_z_OJS_T_selected, {
x: "last",
y: 1,
"stroke": "red"
})
,
Plot.text(summary_tt_z_OJS_T_selected,
Plot.selectFirst({
x: 11,
y: 1,
text: d => d.cohort
})
)
],
facet: {
data: summary_tt_z_OJS_T_selected,
x: "ind"
}
}) write.csv(summary_tt, './models/cmrNN_OB/tt/dataOut/xiaowei/summary_tt.csv')
write.csv(summary_tt_z, './models/cmrNN_OB/tt/dataOut/xiaowei/summary_tt_z.csv')
# #Output input data to Xiaowei directory. Turn to TRUE to output.
# Output happens here: C:/Users/bletcher/OneDrive - DOI/projects/wbBook_quarto_targets/modelsCMR_NN_OB_outputData.qmd
# if(FALSE) {
#
# for (i in seq_along(eh)){
# write.csv(eh[[i]], file = paste0('./models/cmrNN_OB/tt/dataOut/xiaowei/',
# names(eh)[i],
# ".csv"),
# row.names = F)
# }
# }Model with phi and p for each season_isYOY (YOY = young-of-year, isYOY = juvenile or adult) combination. Season is hierarchical over time. There are no data for the isYOY = TRUE, season = 2 state.
Probability of survival (phi) model structure:
phi[yoy, s] <- betaInt[yoy, s]
Probability of capture (p) model structure:
p[yoy, s] <- betaP[yoy, s]
Model code is in ./models/cmrNN_OB/ss/modelCMR_ss_NN_OB_functionsToSource.R
Model is run ‘by hand’ in ./models/cmrNN_OB/ss/modelCMR_ss_NN_OB_makeFile.R
Model runs:
# Following https://oliviergimenez.github.io/bayesian-cr-workshop/worksheets/4_demo.html
#
load('./models/cmrNN_OB/ss/runsOut/ss_NN_OB_mostRecent.RData')
out_ss_NN_OB <- d
#Input data
eh <- tar_read(eh_OB_2002_2014_target)In the model code, a value of 1 for z or in gamma or omega signifies the individual is alive and a value of 2 signifies the individual is dead. y[i,j] is the observed data (encounter history file). isYOY = 1 signifies YOY = TRUE (juvenile), and isYOY = 2 signifies YOY = FALSE (adult)
out_ss_NN_OB$modelCode{
{
delta[1] <- 1
delta[2] <- 0
for (i in 1:N) {
for (t in first[i]:(last[i] - 1)) {
gamma[1, 1, t, i] <- phi[i, t]
gamma[1, 2, t, i] <- 1 - phi[i, t]
gamma[2, 1, t, i] <- 0
gamma[2, 2, t, i] <- 1
omega[1, 1, t, i] <- 1 - p[i, t]
omega[1, 2, t, i] <- p[i, t]
omega[2, 1, t, i] <- 1
omega[2, 2, t, i] <- 0
}
}
for (i in 1:N) {
for (t in first[i]:(last[i] - 1)) {
logit(phi[i, t]) <- phiInt[isYOY_DATA[i, t],
seasons[t]]
logit(p[i, t]) <- pInt[isYOY_DATA[i, t], seasons[t]]
}
}
for (yoy in 1:2) {
for (s in 1:4) {
phiInt[yoy, s] ~ T(dnorm(0, sd = 1/0.667), -3.5,
3.5)
pInt[yoy, s] ~ dunif(0, 1)
}
}
for (i in 1:N) {
z[i, first[i]] ~ dcat(delta[1:2])
for (j in first[i]:(last[i] - 1)) {
z[i, j + 1] ~ dcat(gamma[z[i, j], 1:2, j, i])
yDATA[i, j + 1] ~ dcat(omega[z[i, j + 1], 1:2,
j, i])
}
}
}
}
| nIter | nBurnin | nChains | thinRate |
|---|---|---|---|
| 5000 | 2000 | 2 | 5 |
[1] “Run time = 38.594 mins”
priors <- runif(out_ss_NN_OB$runData$nIter * out_ss_NN_OB$runData$nChains, 0, 1)
MCMCtrace(object = out_ss_NN_OB$mcmc,
#ISB = FALSE,
#exact = TRUE,
params = c("phiInt", "pInt"),
pdf = FALSE,
priors = priors)





#|
MCMCplot(object = out_ss_NN_OB$mcmc, params = c("phiInt"))
MCMCplot(object = out_ss_NN_OB$mcmc, params = c("pInt"))
Summary values
#|
(summary_ss <- MCMCsummary(object = out_ss_NN_OB$mcmc, params = c("phiInt", "pInt"), round = 3) %>%
rownames_to_column(var = "var")) var mean sd 2.5% 50% 97.5% Rhat n.eff
1 phiInt[1, 1] 1.356 0.134 1.111 1.347 1.660 1.00 298
2 phiInt[2, 1] 1.204 0.197 0.851 1.191 1.645 1.01 282
3 phiInt[1, 2] -0.008 1.409 -2.749 0.006 2.660 1.00 1200
4 phiInt[2, 2] 0.299 0.073 0.162 0.297 0.441 1.05 327
5 phiInt[1, 3] 1.458 0.127 1.218 1.454 1.713 1.01 368
6 phiInt[2, 3] 0.586 0.098 0.412 0.583 0.795 1.05 363
7 phiInt[1, 4] 1.291 0.116 1.067 1.288 1.507 1.04 424
8 phiInt[2, 4] 0.737 0.133 0.497 0.730 1.005 1.01 298
9 pInt[1, 1] 0.773 0.093 0.585 0.774 0.958 1.02 402
10 pInt[2, 1] 0.929 0.060 0.785 0.945 0.997 1.04 591
11 pInt[1, 2] 0.498 0.290 0.031 0.484 0.974 1.00 1200
12 pInt[2, 2] 0.731 0.088 0.556 0.732 0.904 1.01 549
13 pInt[1, 3] 0.405 0.080 0.254 0.406 0.562 1.00 636
14 pInt[2, 3] 0.548 0.097 0.349 0.552 0.742 1.01 469
15 pInt[1, 4] 0.022 0.019 0.001 0.017 0.073 1.00 489
16 pInt[2, 4] 0.117 0.078 0.006 0.103 0.295 1.01 527
# To get the mMCMCSummaryRMNA funcion which I adapted to deal with NAs
source('./models/cmrNN_OB/ss/modelCMR_ss_NN_OB_functionsToSource.R')
summary_ss_z0 <- MCMCSummaryRMNA(object = out_ss_NN_OB$mcmc, params = c("z"), round = 3) %>%
rownames_to_column(var = "var") |>
mutate(
ind0 = str_match(var, "([0-9]+), ([0-9]+)")[,2],
t0 = str_match(var, "([0-9]+), ([0-9]+)")[,3],
ind = as.numeric(ind0),
t = as.numeric(t0)
) |>
dplyr::select(-c(t0, ind0))
summary_ss_phi0 <- MCMCSummaryRMNA(object = out_ss_NN_OB$mcmc, params = c("phiInt"), round = 3) %>%
rownames_to_column(var = "var") |>
mutate(
ind0 = str_match(var, "([0-9]+), ([0-9]+)")[,2],
t0 = str_match(var, "([0-9]+), ([0-9]+)")[,3],
ind = as.numeric(ind0),
t = as.numeric(t0)
) |>
dplyr::select(-c(t0, ind0))
summary_ss_p0 <- MCMCSummaryRMNA(object = out_ss_NN_OB$mcmc, params = c("pInt"), round = 3) %>%
rownames_to_column(var = "var") |>
mutate(
ind0 = str_match(var, "([0-9]+), ([0-9]+)")[,2],
t0 = str_match(var, "([0-9]+), ([0-9]+)")[,3],
ind = as.numeric(ind0),
t = as.numeric(t0)
) |>
dplyr::select(-c(t0, ind0))Add other variables to summary values
#|
ehLong <-
eh$eh |>
as.data.frame() |>
rownames_to_column("ind0") |>
pivot_longer(starts_with("ais")) |>
mutate(
t = as.numeric(str_match(name, "([0-9]+)")[,1]),
enc = value,
ind = as.numeric(ind0)
) |>
dplyr::select(-c(name, value, ind0))
firstLong <- eh$first |>
as_tibble() |>
rownames_to_column("ind") |>
mutate(ind = as.numeric(ind)) |>
rename(first = value)
lastLong <- eh$last |>
as_tibble() |>
rownames_to_column("ind") |>
mutate(ind = as.numeric(ind)) |>
rename(last = value)
cohortLong <- eh$cohort |>
as_tibble() |>
rownames_to_column("ind") |>
mutate(ind = as.numeric(ind))
#### z
summary_ss_z <- summary_ss_z0 |>
left_join(ehLong) |>
mutate(
meanM1 = mean - 1,
pSurv = ifelse(meanM1 == 0, 1, 1 - meanM1),
enc8 = ifelse(enc == 1, 0.8, 0)
) |>
left_join(firstLong) |>
left_join(lastLong) |>
left_join(cohortLong)
##### p
summary_ss_p <- summary_ss_p0 |>
left_join(ehLong) |>
mutate(
meanM1 = mean - 1,
pSurv = ifelse(meanM1 == 0, 1, 1 - meanM1),
enc8 = ifelse(enc == 1, 0.8, 0)
) |>
left_join(firstLong) |>
left_join(lastLong) |>
left_join(cohortLong)
#### phi
summary_ss_phi <- summary_ss_phi0 |>
left_join(ehLong) |>
mutate(
meanM1 = mean - 1,
pSurv = ifelse(meanM1 == 0, 1, 1 - meanM1),
enc8 = ifelse(enc == 1, 0.8, 0)
) |>
left_join(firstLong) |>
left_join(lastLong) |>
left_join(cohortLong)if(!exists("np")) {
np <-import("numpy")
}
xP0 <- np$load("./data/in/fromXioawei/p_est.npy")
xP <- xP0 |>
as.data.frame() |>
rowid_to_column("ind") |>
pivot_longer(cols = starts_with("V")) |>
separate_wider_delim(name, delim = "V", names = c("d", "t")) |>
dplyr::select(-d) |>
mutate(t = as.numeric(t)) |>
rename(xP = value)
xPhi0 <- np$load("./data/in/fromXioawei/phi_est.npy")
xPhi <- xPhi0 |>
as.data.frame() |>
rowid_to_column("ind") |>
pivot_longer(cols = starts_with("V")) |>
separate_wider_delim(name, delim = "V", names = c("d", "t")) |>
dplyr::select(-d) |>
mutate(t = as.numeric(t))|>
rename(xPhi = value)Plot Xiaowei’s results
ggplot(xP, aes(t, xP)) +
geom_point(alpha = 0.01)
ggplot(xP, aes(t, xP)) +
geom_jitter(alpha = 0.01)
ggplot(xPhi, aes(t, xPhi)) +
geom_jitter(alpha = 0.01)
summary_xPhi <- xPhi |>
summarize(
meanPhi = mean(xPhi),
sdPhi = sd(xPhi)
)
summary_xP <- xP |>
summarize(
meanP = mean(xP),
sdP = sd(xP)
)summary_ss_phi_x <- summary_ss_phi0 |>
left_join(xPhi)Observable interactivity
ojs_define(numTags_ss_OJS = dim(eh$tags)[1])
ojs_define(summary_ss_z_OJS = (summary_ss_z))
ojs_define(summary_ss_phi_x_OJS = (summary_ss_phi_x))Select one or more individuals :
viewof selectInd_ss = Inputs.select([...Array(numTags_ss_OJS).keys()], {
label: "Which fish?",
value: 1,
step: 1,
multiple: true
})
summary_ss_z_OJS_T = transpose(summary_ss_z_OJS)
summary_ss_z_OJS_T_selected = summary_ss_z_OJS_T.filter((d) =>
selectInd_ss.includes(d.ind)
)
summary_ss_phi_x_OJS_T = transpose(summary_ss_phi_x_OJS)
summary_ss_phi_x_OJS_T_selected = summary_ss_phi_x_OJS_T.filter((d) =>
selectInd_ss.includes(d.ind)
)Black dots/line is estimated probability of being alive and orange dots are encountered yes (y = 0.8)/no (y = 0). The green line is the first observation and the red line is the last possible observation. The number in the upper right corner of each panel is the fish’s cohort.
Plot.plot({
width: width,
height: 350,
inset: 10,
color: {
scheme: "greys"
},
x: { label: "Age/season combination" },
y: { label: "Probability of alive" },
marks: [
Plot.frame(),
Plot.dot(summary_ss_z_OJS_T_selected, {
x: "t",
y: "pSurv"
}),
Plot.dot(summary_ss_z_OJS_T_selected, {
x: "t",
y: "enc8",
fill: "orange"
}),
Plot.line(summary_ss_z_OJS_T_selected, {
x: "t",
y: "pSurv"
}),
Plot.ruleX(summary_ss_z_OJS_T_selected, {
x: "first",
y: 1,
"stroke": "green"
}),
Plot.ruleX(summary_ss_z_OJS_T_selected, {
x: "last",
y: 1,
"stroke": "red"
})
,
Plot.text(summary_ss_z_OJS_T_selected,
Plot.selectFirst({
x: 11,
y: 1,
text: d => d.cohort
})
)
],
facet: {
data: summary_ss_z_OJS_T_selected,
x: "ind"
}
})Plot.plot({
width: width,
height: 350,
inset: 10,
color: {
scheme: "greys"
},
x: { label: "Age/season combination" },
y: { label: "Probability of survival" },
marks: [
Plot.frame(),
Plot.dot(summary_ss_phi_x_OJS_T_selected, {
x: "t",
y: "mean"
}),
Plot.dot(summary_ss_phi_x_OJS_T_selected, {
x: "t",
y: "enc8",
fill: "orange"
}),
Plot.line(summary_ss_phi_x_OJS_T_selected, {
x: "t",
y: "mean"
}),
Plot.dot(summary_ss_phi_x_OJS_T_selected, {
x: "t",
y: "xPhi"
}),
Plot.line(summary_ss_phi_x_OJS_T_selected, {
x: "t",
y: "xPhi"
}),
Plot.ruleX(summary_ss_phi_x_OJS_T_selected, {
x: "first",
y: 1,
"stroke": "green"
}),
Plot.ruleX(summary_ss_phi_x_OJS_T_selected, {
x: "last",
y: 1,
"stroke": "red"
})
,
Plot.text(summary_ss_phi_x_OJS_T_selected,
Plot.selectFirst({
x: 11,
y: 1,
text: d => d.cohort
})
)
],
facet: {
data: summary_ss_phi_x_OJS_T_selected,
x: "ind"
}
}) write.csv(summary_ss, './models/cmrNN_OB/ss/dataOut/xiaowei/summary_ss.csv')
write.csv(summary_ss_z, './models/cmrNN_OB/ss/dataOut/xiaowei/summary_ss_z.csv')
# #Output input data to Xiaowei directory. Turn to TRUE to output.
# Output happens here: C:/Users/bletcher/OneDrive - DOI/projects/wbBook_quarto_targets/modelsCMR_NN_OB_outputData.qmd
# if(FALSE) {
#
# for (i in seq_along(eh)){
# write.csv(eh[[i]], file = paste0('./models/cmrNN_OB/tt/dataOut/xiaowei/',
# names(eh)[i],
# ".csv"),
# row.names = F)
# }
# }